package com.utils;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class UrlExtractor {

	public static void main(String[] args) {
		
	}
	
	public static Set<String> getURLs(String content, String url){
		//下面这句可能会增加服务器负担，所以首先注释掉
		content = content.replaceAll("<script.*?</script>", "").replaceAll("<style>.*?</style>", "");
		//使用set用来除重，
		Set<String> set = new HashSet<String>();
		try {

			Pattern p = Pattern.compile("href *= *[^>]+>(.*?)</",Pattern.CASE_INSENSITIVE);
			Matcher m = p.matcher(content);
			while (m.find()) {
				String content2 = m.group();

				Pattern p2 = Pattern.compile("href *= *[\"']?([^\"'> ]+)[\"']?",Pattern.CASE_INSENSITIVE);
				Matcher m2 = p2.matcher(content2);

				while (m2.find()) {
					String detUrl = m2.group(1);
					if (detUrl.indexOf(":") >= 0) {
						if (detUrl.startsWith("http:")) {
							set.add(constructUrl(detUrl, url));
						}
					} else {
						set.add(constructUrl(detUrl, url));
					}
				}
			}
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
		return set;
	}

	public static String constructUrl(String link, String base)
			throws MalformedURLException {
		link = link.replaceAll("&amp;", "&");
		String path;
		boolean modified;
		boolean absolute;
		int index;
		URL url; // constructed URL combining relative link and base

		//Relative links starting with ?
		if (('?' == link.charAt(0))) { 
			// if any
			if (-1 != (index = base.lastIndexOf('?')))
				base = base.substring(0, index);
			url = new URL(base + link);
		} else
			url = new URL(new URL(base), link);
		path = url.getFile();
		modified = false;
		absolute = link.startsWith("/");
		if (!absolute) { 
			while (path.startsWith("/.")) {
				if (path.startsWith("/../")) {
					path = path.substring(3);
					modified = true;
				} else if (path.startsWith("/./") || path.startsWith("/.")) {
					path = path.substring(2);
					modified = true;
				} else
					break;
			}
		}
		//处理反斜杠，类似于"/\"
		while (-1 != (index = path.indexOf("/\\"))) {
			path = path.substring(0, index + 1) + path.substring(index + 2);
			modified = true;
		}
		if (modified)
			url = new URL(url, path);

		return url.toString();
	}
}
